package com.tim.test.download;

import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import org.apache.commons.lang3.RandomUtils;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.junit.Test;

import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.charset.MalformedInputException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.stream.Collectors;

/**
 * jQuery选择器规则，https://www.runoob.com/jquery/jquery-ref-selectors.html：
 * 	.选择class，#选择id，[]选择属性，:contains(第)内容包含‘第’
 * 	p,h2；同时选择p、h2
 * 	div.main；同时满足
 * 	div .comment；子孙
 * 	div>p；子级
 * 	div~h1；兄弟
 */
public class Xniuyanxs {
	// 解析地址
	String firstPage = "https://www.xniuyanxs.cc/html/nl9j7M4K/2.html";
	// 链接url前缀
	String urlPre = "https://www.xniuyanxs.cc";
	// 内容 父元素；包含导航信息
	String contentContainerCss = "#htmlContent";
	// 内容 目标元素
	String contentElementCss = "#nr";
	// 标题
	String titleCss = "#content h1";
	// 下一页
	String nextPageCss = "#linkNext";
	// 下一章；为空不处理
	String nextPageCss2 = "";
	// 是否执行页面js
	int executeJs = 1;
	// 不解析多子元素的内容 目标元素
	int skipMultiTag = 0;
	/*******以下不改********/
	// 目标文件夹
	String dir = "D:/个人资料/书籍/novel/";
	String pageFileName = "pageFileName"+ RandomUtils.nextInt()+".txt";
	String chapterTitle ="";
	Charset charset = StandardCharsets.UTF_8;
	WebClient webClient;

	@Test
	public void downloadByPage() throws Exception {
		if(StringUtils.isBlank(urlPre)){
			urlPre = firstPage.substring(0, firstPage.lastIndexOf("/")+1);
		}

		File dirFile = new File(dir);
		if(!dirFile.exists()){
			dirFile.mkdirs();
		}

		File file = new File(dir+"/"+pageFileName);
		if(!file.exists()){
			file.createNewFile();
		}
		downloadByPage(firstPage, urlPre);
		doHandle(pageFileName);
		System.out.println("SUCCESS");
		if (webClient!=null){
			webClient.close();
		}
	}
	private Object downloadByPage(String href, String urlPre) throws Exception{

		System.out.println(href);
		Document document;
		if(executeJs==1){
			String html = executeJs(href);
			document = Jsoup.parse(new ByteArrayInputStream(html.getBytes(charset)), charset.name(), "");
		}else {
			document = Jsoup.connect(href).get();
		}
		// 初筛
		Elements containers = document.body().select(contentContainerCss);

		/* step1，获取本页内容，追加写入文档*/
		StringBuilder builder = new StringBuilder();
		if(StringUtils.isNotBlank(titleCss)){
			Elements titleElements = document.body().select(titleCss);
			if(!chapterTitle.equals(titleElements.first().text())){
				chapterTitle = titleElements.first().text();
				builder.append(chapterTitle).append("\n");
			}
		}
		// 选中目标元素
		Elements targetTags = containers.select(contentElementCss);
		// 获取内容
		for (Element targetTag : targetTags){
			if(targetTag.childrenSize()>0 && skipMultiTag==1){
				continue;
			}
			String text = getText(targetTag).trim();
			builder.append(text).append("\n");
		}
		String content = builder.toString()
				.replaceAll("\n+", "\n");


		Charset charset1 = document.charset();
		Files.write(Paths.get(dir,pageFileName),
				content.getBytes(charset),
				StandardOpenOption.APPEND
		);


		/* step2，获取下一页链接；无则终止*/
		Elements nextPages = document.body().select(nextPageCss);
		Element nextPage = nextPages.stream()
				.filter(e->{
					String text = e.text();
					return StringUtils.isNotBlank(text) && text.trim().contains("下一");
				}).findFirst().orElse(null);
		if(nextPage!=null){
			String nextPageHref = nextPage.attr("href");
			nextPageHref = nextPageHref.startsWith("http") ? nextPageHref : urlPre + nextPageHref;
			return downloadByPage(nextPageHref, urlPre);
		}else if(StringUtils.isNotBlank(nextPageCss2)){
			Elements nextPages2 = document.body().select(nextPageCss2);
			Element nextPage2 = nextPages2.stream()
					.filter(e->{
						String text = e.text();
						return StringUtils.isNotBlank(text) && text.trim().contains("下一");
					}).findFirst().orElse(null);
			if(nextPage2!=null){
				String nextPageHref = nextPage2.attr("href");
				nextPageHref = nextPageHref.startsWith("http") ? nextPageHref : urlPre + nextPageHref;
				return downloadByPage(nextPageHref, urlPre);
			}
		}
		return null;
	}

	private String getText(Element child) {
		return child.html()
				.replaceAll("<br>", "\n")
				.replaceAll("\n+", "\n")
				.replaceAll("&nbsp;", " ");

	}

	private String executeJs(String url){
		if(webClient==null){
			webClient = new WebClient(BrowserVersion.CHROME);//新建一个模拟谷歌Chrome浏览器的浏览器客户端对象

			webClient.getOptions().setThrowExceptionOnScriptError(false);//当JS执行出错的时候是否抛出异常, 这里选择不需要
			webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);//当HTTP的状态非200时是否抛出异常, 这里选择不需要
			webClient.getOptions().setActiveXNative(false);//不启用ActiveX
			webClient.getOptions().setCssEnabled(false);//是否启用CSS, 因为不需要展现页面, 所以不需要启用
			webClient.getOptions().setJavaScriptEnabled(true); //很重要，启用JS
			webClient.getOptions().setDownloadImages(false);//不下载图片
			webClient.setAjaxController(new NicelyResynchronizingAjaxController());//很重要，设置支持AJAX

		}
		HtmlPage page = null;
		try {
			page = webClient.getPage(url);//尝试加载上面图片例子给出的网页
		} catch (Exception e) {
			e.printStackTrace();
		}

		webClient.waitForBackgroundJavaScript(20*1000);//异步JS执行需要耗时,所以这里线程要阻塞30秒,等待异步JS执行结束

		charset = page.getCharset();
		String pageXml = page.asXml();//直接将加载完成的页面转换成xml格式的字符串
//		Document parse = Jsoup.parse(pageXml);
//		System.out.println(parse);
		return pageXml;
	}

	@Test
	public void handleFile() throws IOException {
		String filePath = "pageFileName1502337489.txt";
		doHandle(filePath);
	}

	private void doHandle(String filePath) throws IOException {
		filePath = filePath.contains(":")?filePath:dir+"pageFileName1502337489.txt";
		String[] filters = new String[]{
				"正在手打中",
				"近期更新缓慢"
		};
		Path path = Paths.get(filePath);
		List<String> list;
		try{
			list = Files.readAllLines(path);
		}catch (MalformedInputException e){
			list = Files.readAllLines(path, Charset.forName("gb2312"));
		}
		String content = list.stream()
				.filter(e -> !StringUtils.startsWithAny(e.trim(),filters))
				.map(String::trim)
				.collect(Collectors.joining("\n"));
		content = content.replaceAll("((<p>)|(</p>)|(<br>)|(<br/>))", "")
				.replaceAll("((<b>)|(</b>))", "")
				.replaceAll("　　", "")
				.replaceAll("・", ".")
				.replaceAll(" {2,}", "  ")
				.replaceAll("单?M", "单芃");
		Files.write(path,
				content.getBytes());
	}
}
